from pprint import pprint
from spidertools.utils.xpath_utils import get_alltext
from extractors.base_extractor import BaseExtractor
import re
from scrapy.selector import Selector
import pymongo
from spidertools.utils.snippets import combine_two_dict

# http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=5ed71abb-fc4e-4cb7-8498-3a7e86f341f0&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1
class MongoDb(object):

    def __init__(self):
        self.my_cli = pymongo.MongoClient("mongodb://127.0.0.1:27017")
        self.my_db = self.my_cli["construction"]
        self.my_col = self.my_db["project_html"]
        self.select_url = ["http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=439a1778e2204ca4a424e68cd66ba6c8&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1",
"http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=7d3a03683c4c42dca425acac84ba486d&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1",
"http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=67b134ab833442978943d8d7c93cf0bd&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1",
"http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=4af540312c3a4a169f4d0b23d7cef08e&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1",
"http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=6d892185b2cf49b9b0fe0b0382a351c4&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1",
"http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=f300e67da6ff4b6fb1a0a900f0eb1d6a&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1",
"http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=c2d559e579494d4291bb108a729403d4&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1",
"http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=c374d4cd39b14d038ae114d783786795&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1",]

    @property
    def query_condition(self):
        # return {"source_type": "广东省全国公共资源交易平台", "公告类型": "招标公告"}
        #
        # return {"origin_url": "http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=ddee0df2-98ee-485c-a42a-8b4b0a7d7778&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1"}
        return {"origin_url": "http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=e53d34dc-32a9-4578-b057-d33cd0f65164&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1"}
    # def query_condition(self, url):
    #     return {"origin_url": f"{url}"}

    def get_db_info(self):
        # for url in self.select_url:
        #     for info in self.my_col.find(self.query_condition(url)):
        #         yield info
        for info in self.my_col.find(self.query_condition):
            yield info


# 广东省全国公共资源交易平台
class GuangDongShengQuanGuoGongGongZiYuanJiaoYiPingTai(BaseExtractor):

    def __init__(self, info_dict):

        super(GuangDongShengQuanGuoGongGongZiYuanJiaoYiPingTai, self).__init__(info_dict)
        self.reserve_pro_name = self.sel.xpath("//div[@id='bulletinName']/text()").extract_first()
        self.replace_dict_for_text_mdd = {
            "公告日期[:：]{1}": "",
            "网址:":""
        }

    @property
    def check_content_is_html(self):
        is_html = False
        if "</p>" in "".join(self.sel.xpath("//ul[@id='bulletinContent']//text()").extract()):
            f = open("乱码的url", 'a', encoding="utf-8")
            f.write(self.info_dict["origin_url"] + "\n")
            f.close()
            is_html = True
        print(self.info_dict["origin"])
        return is_html

    def check_purchase(self):
        flag = 0
        if "采购" in self.info_dict["公告标题"]:
            flag = 1
        return flag

    def checkIfNeedParse(self):
        '''
        判断是不是一个有效的html
        '''
        if self.info_dict["公告类型"] == "招标公告":
            flag = 1
            text_list = self.sel.xpath("//ul[@id='bulletinContent']/text()[1]").extract()[0]
            text = text_list.strip()
            if text.startswith("详见"):
                flag = 0
            return flag

    def common_replace_word(self, text):
        '''
        对网页进行一些词条替换
        '''
        for k, v in self.replace_dict_for_text_mdd.items():
            text = re.sub(k, v, text)
        return text

    def common_text_parse(self, xpath="//ul[@id='bulletinContent']//p|//ul[@id='bulletinContent']//h1"):
        ph_nodes = self.sel.xpath(xpath)
        p_text_list = []
        for index, p_node in enumerate(ph_nodes):
            line_content = "".join(list(map(lambda x: x.strip(), p_node.xpath(".//text()").extract()))).strip()
            if line_content:
                if line_content in ["附件一：", "附件一:"]:
                    break
                elif line_content[-1] in [":", "："]:
                    try:
                        line_content = line_content + "".join(ph_nodes[index+1].xpath(".//text()").extract()).strip()
                    except IndexError:
                        print(self.info_dict["origin_url"])
                elif line_content.replace(" ", "").startswith("日期："):
                    continue
                print(line_content)
                p_text_list.append(line_content)

        for i in range(len(p_text_list)):
            p_text_list[i] = self.common_replace_word(p_text_list[i])
        output_dict = self.texts_matchine_parse(p_text_list)
        return output_dict

    # h1-------->http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=5244b9c011f8420686cb20c27ffd1ab7&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1
    # p--------->http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=e53d34dc-32a9-4578-b057-d33cd0f65164&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1
    def get_cut_node_index(self):
        exist_xpath = ["//ul[@id='bulletinContent']//p",
                       "//ul[@id='bulletinContent']//h1"]
        for index1, xp in enumerate(exist_xpath):
            ph_nodes = self.sel.xpath(xp)
            for index2, p_node in enumerate(ph_nodes):
                line_content = "".join(list(map(lambda x: x.strip(), p_node.xpath(".//text()").extract()))).strip()
                if line_content in ["附件一：", "附件一:"]:
                    return index1 + 1, index2 + 1

        return None

    def get_infos_from_otherway(self, result, html=None):
        if "工程基本信息" not in result.keys():
            result["工程基本信息"] = {}

        if not result["工程基本信息"].get("项目名称"):
            result["工程基本信息"]["项目名称"] = self.reserve_pro_name

        if "工程招标信息" not in result.keys():
            result["工程招标信息"] = {}

        if not result["工程招标信息"].get("标段名称"):
            result["工程招标信息"]["标段名称"] = result["工程基本信息"]["项目名称"]
        return result

    def common_table_parse(self):
        output_dict = {}

        # ---------------------------------------------------------------表格的情况2_start-------------------------------------------------------------------
        if self.get_cut_node_index():
            h_or_p, index = self.get_cut_node_index()
            v = ""
            if h_or_p == 1:
                v = self.sel.xpath(f"//ul[@id='bulletinContent']//p[{index}]").extract_first()
            elif h_or_p == 2:
                v = self.sel.xpath(f"//ul[@id='bulletinContent']//h1[{index}]").extract_first()
            v = v.replace(" ", "&nbsp;")
            self.sel = Selector(text=self.html.split(v)[0])

        # ---------------------------------------------------------------表格的情况2_end-------------------------------------------------------------------
        tables = self.sel.xpath("//ul[@id='bulletinContent']//table")

        for table in tables:
            # ---------------------------------------------------------------表格的情况1_start-------------------------------------------------------------------
            # http://bs.gdggzy.org.cn/osh-web/project/projectbulletin/toBullDetail?bulletinId=e8580aa7-812c-4f47-b1e9-51d8db2add95&tradeTypeId=Construction&tradeItemId=gc_res_bulletin&queryType=1
            is_break = False
            for tr in table.xpath(".//tr"):
                tr_content = "".join(list(map(lambda x: x.strip(), tr.xpath(".//text()").extract())))
                if tr_content in ["招标议程安排表"]:
                    is_break = True
                    break
            if is_break:
                break
            # ---------------------------------------------------------------表格的情况1_end-------------------------------------------------------------------










            result = self.table_extract_machine.run_table(table)
            output_dict = combine_two_dict(output_dict, result)
        return output_dict


if __name__ == '__main__':
    m = MongoDb()
    for info in m.get_db_info():
        print(info["origin_url"])
        g = GuangDongShengQuanGuoGongGongZiYuanJiaoYiPingTai(info_dict=info)
        pprint(g.output_extractor_dict(use_new=True))















